Linden 2016

Import simulation results

dataname <- "Linden_2015"

# import simulation results
sim_results <- readRDS(paste0("sim_results/simulation_results_final_", dataname,"_", n_rep, ".rds"))

# define classifiers
classifiers <- names(sim_results)
classifiers <- classifiers[!(classifiers %in% c("outcome", "true_e"))]
classifiers_clean <- c("adaBoost", "bagging" ,"kNN", "LDA", "Multinimial logit (glmnet)", "Multinimial logit (nnet)", "MLPC", "Naive Bayes (Bernulli)", "Naive Bayes (Gaussian)", "Probability forest (grf)", "Probability forest (ranger)", "QDA", "SVM", "XGBoost")

Compute average evaluation metrics

The predicted propensity score are evaluated using the Brier score, the RMSE and the logistic loss.

bs_mat <- matrix(NA, nrow = n_rep, ncol = length(classifiers), dimnames = list(NULL, classifiers))
ll_mat <- matrix(NA, nrow = n_rep, ncol = length(classifiers), dimnames = list(NULL, classifiers))
rmse_mat <- matrix(NA, nrow = n_rep, ncol = length(classifiers), dimnames = list(NULL, classifiers))

bs_mat_oracle <- matrix(NA, nrow=n_rep, ncol = 1)

n=length(sim_results$outcome[[1]]$W)
for (i in 1:n_rep){
  bs_mat_oracle[i] <- brier_score(probabilities=sim_results[["true_e"]][[i]], 
                      outcome=sim_results[["outcome"]][[i]]$W, 
                      binary = FALSE)

  for (c in seq_along(classifiers)) {
    classifier <- classifiers[c]

    bs_mat[i, c] <- brier_score(probabilities=sim_results[[classifier]][[i]], 
                      outcome=sim_results[["outcome"]][[i]]$W, 
                      binary = FALSE)
    
    rmse_mat[i, c] <- sqrt(mean(rowMeans((sim_results[[classifier]][[i]] - sim_results[["true_e"]][[i]])^2, na.rm = TRUE), na.rm = TRUE))

    ll_mat[i, c] <- cross_entropy(sim_results[[classifier]][[i]], sim_results[["outcome"]][[i]]$W)
  }
}

Reshape and aggregate results

  1. Brier score
bs_long <- bs_mat %>%
  as.data.frame() %>%
  pivot_longer(cols = everything())  %>%
  # Separate the 'name' into 'classifier' and 'version' columns
  mutate(version = case_when(
           str_detect(name, "^ovr_") ~ "one-vs-rest",
           str_detect(name, "^ovo_") ~ "one-vs-one",
           TRUE ~ "multi-class"
         ),
         name = gsub("ovr_|ovo_", "", name)
         )
bs_wide <- bs_long %>%
  group_by(version, name) %>% 
  summarise(value = round(mean(value), 6)) %>% 
  spread(key = version, value = value) 
`summarise()` has grouped output by 'version'. You can override using the `.groups` argument.
kable(bs_wide, caption = "Brier Score")
Brier Score
name multi-class one-vs-one one-vs-rest
adaboost NA 0.231426 0.233096
bagging NA 0.236621 0.243976
knn 0.225868 0.225448 0.218266
lda 0.213472 0.213510 0.213479
logit 0.213337 0.247198 0.213417
logit_nnet 0.213429 0.213533 0.213475
mlpc 0.213293 0.216825 0.216747
nb_bernulli 0.214456 0.214547 0.214486
nb_gaussian 0.214450 0.214495 0.214475
probability_forest 0.216493 0.215474 0.216540
qda 0.226034 0.226233 0.224151
ranger 0.235241 0.217772 0.217983
svm NA 0.220312 0.220447
xgboost NA 0.229347 0.218846
  1. RMSE
rmse_long <- rmse_mat %>%
  as.data.frame() %>%
  pivot_longer(cols = everything())  %>%
  # Separate the 'name' into 'classifier' and 'version' columns
  mutate(version = case_when(
           str_detect(name, "^ovr_") ~ "one-vs-rest",
           str_detect(name, "^ovo_") ~ "one-vs-one",
           TRUE ~ "multi-class"
         ),
         name = gsub("ovr_|ovo_", "", name)
         ) 
rmse_wide <- rmse_long %>%
  group_by(version, name) %>% 
  summarise(value = round(mean(value), 4)) %>% 
  spread(key = version, value = value) 
`summarise()` has grouped output by 'version'. You can override using the `.groups` argument.
kable(rmse_wide, caption = "RMSE")
RMSE
name multi-class one-vs-one one-vs-rest
adaboost NA 0.1366 0.1431
bagging NA 0.1543 0.1781
knn 0.1127 0.1103 0.0753
lda 0.0298 0.0299 0.0295
logit 0.0272 0.1857 0.0281
logit_nnet 0.0286 0.0297 0.0293
mlpc 0.0269 0.0650 0.0646
nb_bernulli 0.0427 0.0439 0.0429
nb_gaussian 0.0428 0.0439 0.0428
probability_forest 0.0623 0.0535 0.0628
qda 0.1157 0.1163 0.1063
ranger 0.1505 0.0720 0.0731
svm NA 0.0883 0.0891
xgboost NA 0.1290 0.0787
  1. Logistic loss
ll_long <- ll_mat %>%
  as.data.frame() %>%
  pivot_longer(cols = everything())  %>%
  # Separate the 'name' into 'classifier' and 'version' columns
  mutate(version = case_when(
           str_detect(name, "^ovr_") ~ "one-vs-rest",
           str_detect(name, "^ovo_") ~ "one-vs-one",
           TRUE ~ "multi-class"
         ),
         name = gsub("ovr_|ovo_", "", name)
         ) 
ll_wide <- ll_long %>%
  group_by(version, name) %>% 
  summarise(value = round(mean(value), 4)) %>% 
  spread(key = version, value = value) 
`summarise()` has grouped output by 'version'. You can override using the `.groups` argument.
kable(ll_wide, caption = "Log Loss")
Log Loss
name multi-class one-vs-one one-vs-rest
adaboost NA 1.1482 1.3000
bagging NA 1.1909 1.7804
knn 1.7681 NA 1.0858
lda 1.0605 1.0608 1.0605
logit 1.0600 1.2253 1.0604
logit_nnet 1.0604 1.0609 1.0607
mlpc 1.0597 1.0819 1.0946
nb_bernulli 1.0653 1.0658 1.0657
nb_gaussian 1.0653 1.0655 1.0656
probability_forest 1.0751 1.0697 1.0754
qda 1.1820 NA 1.1716
ranger 1.1746 1.0809 1.0828
svm NA 1.0905 1.0909
xgboost NA 1.1314 1.0837

Export the final results.

# Write bs_wide data frame as an Excel file
write.xlsx(bs_wide, file = paste0("sim_results/results_bs_", dataname, ".xlsx"))

# Write mse_wide data frame as an Excel file
write.xlsx(rmse_wide, file = paste0("sim_results/results_mse_", dataname, ".xlsx"))

Visualization

For the synthetic datasets, the true propensity scores e are known and can be utilized to establish a benchmark for evaluating the Brier score. This reference Brier score calculated using the ground truth e is denoted as BS_oracle. BS_oracle has perfect information of the data and can be seen as a lower bound for the Brier score. A second benchmark for the Brier score is a naïve guess, using equal treatment propensity scores of e = 1/T for every treatment level denoted as BS_naive.

K <- length(unique(sim_results$outcome[[1]]$W)) # number of treatments
BS_naive <- (((1/K)-1)^2 + (K-1)*((1/K)^2))/K # BS_naive
BS_oracle <- mean(bs_mat_oracle) # BS_oracle
  
plot1 <- ggplot(bs_long, aes(x = value, y = fct_rev(name), fill = version)) +
  geom_density_ridges(alpha = 0.9, show.legend = FALSE) +
  geom_vline(xintercept = BS_naive, color="black", linetype = "dashed", size = 0.4) +
  geom_vline(xintercept = BS_oracle, color="black", linetype = "dashed", size = 0.4) +
  scale_fill_paletteer_d("tayloRswift::midnightsBloodMoon", direction = 1, dynamic = FALSE) +
  labs(y="", x="Brier Score", fill="") +
  theme_bw() +
  xlim(0.205, 0.25) +
  scale_y_discrete(labels = rev(classifiers_clean)) +
  facet_wrap(vars(version))
plot1
ggsave(plot1, width = 250, height = 125, units = "mm",
       filename = "plots/dens1.png")

Another way to assess the quality of the predicted propensity scores is to plot them against the ground truth in a scatter plot.

oracle_e <- reshape2::melt(sim_results[["true_e"]][[1]], id.vars = NULL, variable.name = "class", value.name = "true")
for (c in classifiers){
  predicted_e <- reshape2::melt(sim_results[[c]][[1]], id.vars = NULL, variable.name = "class", value.name = "prediction") %>% select(-class)

  plot_data <- cbind(predicted_e, oracle_e)
  
  scatter_plot <- ggplot(data = plot_data,aes(y = prediction, x = true, color=class)) +
    geom_point(alpha = 0.3, size=1) +
    geom_abline(intercept = 0, slope = 1, linetype = "dashed", color = "red") + 
    theme_bw() +
    scale_color_manual(values=c("#D890A0FF", "#282020FF", "#F8B840FF")) +
    labs(
          x = "Predicted Probabilities ê(w)",
          y = "True Probailities e(w)",
          color = "W"
          #title = paste0("Simulation results of ", c)
        ) +
    xlim(0,1) + ylim(0,1)
  
  plot(scatter_plot)
  ggsave(scatter_plot, width = 125, height = 75, units = "mm",
         filename = paste0("plots/truevspred_", c, ".png"))
}

Imbens 2016

Import simulation results

dataname <- "Imbens_2016"

sim_results <- readRDS(paste0("sim_results/simulation_results_final_", dataname,"_", n_rep, ".rds"))

Compute average evaluation metrics

The predicted propensity score are evaluated using the Brier score, the RMSE and the logistic loss.

bs_mat <- matrix(NA, nrow = n_rep, ncol = length(classifiers), dimnames = list(NULL, classifiers))
ll_mat <- matrix(NA, nrow = n_rep, ncol = length(classifiers), dimnames = list(NULL, classifiers))
rmse_mat <- matrix(NA, nrow = n_rep, ncol = length(classifiers), dimnames = list(NULL, classifiers))

bs_mat_oracle <- matrix(NA, nrow=n_rep, ncol = 1)

n=length(sim_results$outcome[[1]]$W)
for (i in 1:n_rep){
  bs_mat_oracle[i] <- brier_score(probabilities=sim_results[["true_e"]][[i]], 
                      outcome=sim_results[["outcome"]][[i]]$W, 
                      binary = FALSE)

  for (c in seq_along(classifiers)) {
    classifier <- classifiers[c]

    bs_mat[i, c] <- brier_score(probabilities=sim_results[[classifier]][[i]], 
                      outcome=sim_results[["outcome"]][[i]]$W, 
                      binary = FALSE)
    
    rmse_mat[i, c] <- sqrt(mean(rowMeans((sim_results[[classifier]][[i]] - sim_results[["true_e"]][[i]])^2, na.rm = TRUE), na.rm = TRUE))

    ll_mat[i, c] <- cross_entropy(sim_results[[classifier]][[i]], sim_results[["outcome"]][[i]]$W)
  }
}

Reshape and aggregate results

  1. Brier score
bs_long <- bs_mat %>%
  as.data.frame() %>%
  pivot_longer(cols = everything())  %>%
  # Separate the 'name' into 'classifier' and 'version' columns
  mutate(version = case_when(
           str_detect(name, "^ovr_") ~ "one-vs-rest",
           str_detect(name, "^ovo_") ~ "one-vs-one",
           TRUE ~ "multi-class"
         ),
         name = gsub("ovr_|ovo_", "", name)
         )
bs_wide <- bs_long %>%
  group_by(version, name) %>% 
  summarise(value = round(mean(value), 4)) %>% 
  spread(key = version, value = value) 
`summarise()` has grouped output by 'version'. You can override using the `.groups` argument.
kable(bs_wide, caption = "Brier Score")
Brier Score
name multi-class one-vs-one one-vs-rest
adaboost NA 0.2320 0.2327
bagging NA 0.2325 0.2368
knn 0.2495 0.2481 0.2265
lda 0.2218 0.2219 0.2219
logit 0.2218 0.2287 0.2217
logit_nnet 0.2218 0.2220 0.2219
mlpc 0.2235 0.2358 0.2344
nb_bernulli 0.2257 0.2257 0.2249
nb_gaussian 0.2256 0.2260 0.2249
probability_forest 0.2213 0.2212 0.2214
qda 0.2260 0.2262 0.2255
ranger 0.2275 0.2234 0.2232
svm NA 0.2231 0.2229
xgboost NA 0.2252 0.2216
  1. RMSE
rmse_long <- rmse_mat %>%
  as.data.frame() %>%
  pivot_longer(cols = everything())  %>%
  # Separate the 'name' into 'classifier' and 'version' columns
  mutate(version = case_when(
           str_detect(name, "^ovr_") ~ "one-vs-rest",
           str_detect(name, "^ovo_") ~ "one-vs-one",
           TRUE ~ "multi-class"
         ),
         name = gsub("ovr_|ovo_", "", name)
         ) 
rmse_wide <- rmse_long %>%
  group_by(version, name) %>% 
  summarise(value = round(mean(value), 4)) %>% 
  spread(key = version, value = value) 
`summarise()` has grouped output by 'version'. You can override using the `.groups` argument.
kable(rmse_wide, caption = "RMSE")
RMSE
name multi-class one-vs-one one-vs-rest
adaboost NA 0.1255 0.1287
bagging NA 0.1306 0.1440
knn 0.1735 0.1755 0.1057
lda 0.0735 0.0743 0.0739
logit 0.0727 0.1123 0.0730
logit_nnet 0.0734 0.0744 0.0739
mlpc 0.0842 0.1396 0.1348
nb_bernulli 0.1038 0.1045 0.0990
nb_gaussian 0.1036 0.1049 0.0991
probability_forest 0.0702 0.0689 0.0706
qda 0.1041 0.1054 0.1008
ranger 0.1062 0.0837 0.0847
svm NA 0.0802 0.0788
xgboost NA 0.0938 0.0734
  1. Logistic loss
ll_long <- ll_mat %>%
  as.data.frame() %>%
  pivot_longer(cols = everything())  %>%
  # Separate the 'name' into 'classifier' and 'version' columns
  mutate(version = case_when(
           str_detect(name, "^ovr_") ~ "one-vs-rest",
           str_detect(name, "^ovo_") ~ "one-vs-one",
           TRUE ~ "multi-class"
         ),
         name = gsub("ovr_|ovo_", "", name)
         ) 
ll_wide <- ll_long %>%
  group_by(version, name) %>% 
  summarise(value = round(mean(value), 4)) %>% 
  spread(key = version, value = value) 
`summarise()` has grouped output by 'version'. You can override using the `.groups` argument.
kable(ll_wide, caption = "Log Loss")
Log Loss
name multi-class one-vs-one one-vs-rest
adaboost NA 1.1462 1.2067
bagging NA 1.1505 1.2421
knn 4.2125 NA 1.1212
lda 1.0968 1.0973 1.0973
logit 1.0965 1.1282 1.0964
logit_nnet 1.0967 1.0973 1.0970
mlpc 1.1109 1.1692 1.5063
nb_bernulli 1.1351 NA 1.1296
nb_gaussian 1.1349 1.1488 1.1295
probability_forest 1.0945 1.0940 1.0951
qda 1.1377 1.1509 1.1340
ranger 1.1238 1.1043 1.1039
svm NA 1.1024 1.1018
xgboost NA 1.1123 1.0958

Export the final results.

# Write bs_wide data frame as an Excel file
write.xlsx(bs_wide, file = paste0("sim_results/results_bs_", dataname, ".xlsx"))

# Write mse_wide data frame as an Excel file
write.xlsx(rmse_wide, file = paste0("sim_results/results_mse_", dataname, ".xlsx"))

Visualization

K <- length(unique(sim_results$outcome[[1]]$W))
BS_naive <- (((1/K)-1)^2 + (K-1)*((1/K)^2))/K
LL_unif <- -(log(1/K) + log(1-(1/K)))
BS_oracle <- mean(bs_mat_oracle)
  
plot2 <- ggplot(bs_long, aes(x = value, y = fct_rev(name), fill = version)) +
  geom_density_ridges(alpha = 0.9, show.legend = FALSE) +
  geom_vline(xintercept = BS_naive, color="black", linetype = "dashed") +
  geom_vline(xintercept = BS_oracle, color="black", linetype = "dashed") +
  scale_fill_paletteer_d("tayloRswift::midnightsBloodMoon", direction = 1, dynamic = FALSE) +
  labs(y="", x="Brier Score", fill="") +
  theme_bw() +
  xlim(0.215, 0.25) +
  scale_y_discrete(labels = rev(classifiers_clean)) +
  facet_wrap(vars(version))
plot2
ggsave(plot2, width = 250, height = 125, units = "mm",
       filename = "plots/dens2.png")

pi <- sim_results$true_e[[1]]

bs_oracle <- brier_score(probabilities = pi, outcome = sim_results[["outcome"]][[1]]$W)
ll_oracle <- cross_entropy(probabilities = pi, outcome = sim_results[["outcome"]][[1]]$W)

predicted_e <- reshape2::melt(sim_results[[c]][[1]], id.vars = NULL, variable.name = "class", value.name = "prediction")

for (c in classifiers){
  oracle_e <- reshape2::melt(pi, id.vars = NULL, variable.name = "class", value.name = "true") %>% select(true)
  plot_data <- cbind(predicted_e, oracle_e)

  scatter_plot <- ggplot(data = plot_data,aes(y = prediction, x = true, color=class)) +
    geom_point(alpha = 0.3) +
    geom_abline(intercept = 0, slope = 1, linetype = "dashed", color = "red") + 
    geom_smooth(color="red", se = FALSE, linewidth = 0.7)+
    theme_bw() +
    scale_color_paletteer_d("tayloRswift::midnightsBloodMoon", direction = 1, dynamic = FALSE) +
    labs(
          x = "Predicted Probabilities",
          y = "True Probailities",
          title = paste0("Simulation results of ", c)
        ) +
    xlim(0,1) + ylim(0,1)

plot(scatter_plot)
}

Acharky 2023

Import simulation results

dataname <- "Acharky_2023"
n_rep <- 100

sim_results <- readRDS(paste0("sim_results/simulation_results_ovo_", dataname,"_", n_rep, ".rds"))

Compute average evaluation metrics

The predicted propensity score are evaluated using the Brier score and the logistic loss. As for the semi-synthetic dataset the ground truth of the propensity score is not know, the RMSE cannot be calculated.

bs_mat <- matrix(NA, nrow = n_rep, ncol = length(classifiers), dimnames = list(NULL, classifiers))
ll_mat <- matrix(NA, nrow = n_rep, ncol = length(classifiers), dimnames = list(NULL, classifiers))
n=length(sim_results$outcome[[1]]$W)
for (c in seq_along(classifiers)) {
  classifier <- classifiers[c]
  for (i in 1:n_rep){
    bs <- brier_score(sim_results[[classifier]][[i]], sim_results[["outcome"]][[i]]$W, binary = FALSE)
    bs_mat[i, c] <- bs
    
    ll <- cross_entropy(sim_results[[classifier]][[i]], sim_results[["outcome"]][[i]]$W)
    ll_mat[i, c] <- ll
  }
}

Reshape and aggregate results

  1. Brier score
bs_long <- bs_mat %>%
  as.data.frame() %>%
  pivot_longer(cols = everything())  %>%
  # Separate the 'name' into 'classifier' and 'version' columns
  mutate(version = case_when(
           str_detect(name, "^ovr_") ~ "one-vs-rest",
           str_detect(name, "^ovo_") ~ "one-vs-one",
           TRUE ~ "multi-class"
         ),
         name = gsub("ovr_|ovo_", "", name)
         )
bs_wide <- bs_long %>%
  group_by(version, name) %>% 
  summarise(value = round(mean(value), 4)) %>% 
  spread(key = version, value = value) 
`summarise()` has grouped output by 'version'. You can override using the `.groups` argument.
kable(bs_wide, caption = "Brier Score")
Brier Score
name multi-class one-vs-one one-vs-rest
adaboost NA 0.0719 0.0732
bagging NA 0.0721 0.0807
knn 0.1132 0.0750 0.0778
lda 0.0712 0.0712 0.0712
logit 0.0712 0.0712 0.0712
logit_nnet 0.0711 0.0712 0.0712
mlpc 0.0710 0.0710 0.0710
nb_bernulli 0.0713 0.0713 0.0713
nb_gaussian 0.0713 0.0713 0.0713
probability_forest 0.0715 0.0711 0.0715
qda 0.0719 0.0720 0.0720
ranger 0.0727 0.0712 0.0712
svm NA 0.0710 0.0711
xgboost NA 0.0711 0.0780
  1. Logistic loss
ll_long <- ll_mat %>%
  as.data.frame() %>%
  pivot_longer(cols = everything())  %>%
  # Separate the 'name' into 'classifier' and 'version' columns
  mutate(version = case_when(
           str_detect(name, "^ovr_") ~ "one-vs-rest",
           str_detect(name, "^ovo_") ~ "one-vs-one",
           TRUE ~ "multi-class"
         ),
         name = gsub("ovr_|ovo_", "", name)
         ) 
ll_wide <- ll_long %>%
  group_by(version, name) %>% 
  summarise(value = round(mean(value), 4)) %>% 
  spread(key = version, value = value) 
`summarise()` has grouped output by 'version'. You can override using the `.groups` argument.
kable(ll_wide, caption = "Log Loss")
Log Loss
name multi-class one-vs-one one-vs-rest
adaboost NA 2.6336 4.0974
bagging NA 2.6560 20.4284
knn 36.5656 2.8852 13.3819
lda 2.5806 2.5814 2.5810
logit 2.5781 2.5801 2.5788
logit_nnet 2.5727 2.5816 2.5808
mlpc 2.5666 2.5666 2.5668
nb_bernulli 2.5914 2.5925 2.5927
nb_gaussian 2.5910 2.5927 2.5928
probability_forest 2.6062 2.5729 2.6073
qda 2.6460 2.6512 2.6507
ranger 2.6948 2.5816 2.5822
svm NA 2.5676 2.5693
xgboost NA 2.5697 3.1047

Export the final results.

# Write bs_wide data frame as an Excel file
write.xlsx(bs_wide, file = paste0("sim_results/results_bs_", dataname, ".xlsx"))

Visualization

K <- length(unique(sim_results$outcome[[1]]$W))
BS_oracle <- (((1/K)-1)^2 + (K-1)*((1/K)^2))/K
LL_unif <- -(log(1/K) + log(1-(1/K)))

plot3 <- ggplot(bs_long, aes(x = value, y = fct_rev(name), fill = version)) +
  geom_density_ridges(alpha = 0.9, show.legend = FALSE) +
  geom_vline(xintercept = BS_oracle, color="black", linetype = "dashed") +
  scale_fill_paletteer_d("tayloRswift::midnightsBloodMoon", direction = 1, dynamic = FALSE) +
  labs(y="", x="Brier Score", fill="") +
  theme_bw() +
  scale_x_continuous(breaks = c(0.071, 0.0715, 0.072), limits = c(0.0708, 0.0721)) +
  scale_y_discrete(labels = rev(classifiers_clean)) +
  facet_wrap(vars(version))
plot3
ggsave(plot3, width = 250, height = 125, units = "mm",
       filename = "plots/dens3.png")

for (c in classifiers){
  plot(make_calibtation_plot(probabilities = sim_results[[c]][[1]], outcome = sim_results$outcome[[1]]$W, method = c))
}
`summarise()` has grouped output by 'probability_bin'. You can override using the `.groups` argument.

LS0tDQp0aXRsZTogIkV2YWx1YXRpb24gb2YgU2ltdWxhdGlvbiByZXN1bHRzIg0KYXV0aG9yOiAiTWFyZW4gQmF1bWfDpHJ0bmVyIg0Kb3V0cHV0OiANCiAgaHRtbF9ub3RlYm9vazoNCiAgICB0b2M6IHRydWUNCiAgICB0b2NfZmxvYXQ6IHRydWUNCiAgICBjb2RlX2ZvbGRpbmc6IHNob3cNCi0tLQ0KDQpgYGB7ciwgbWVzc2FnZT1GQUxTRSwgZWNobz1GQUxTRSwgd2FybmluZz1GQUxTRX0NCnNldC5zZWVkKDQyKSAjIHNldCBzZWVkDQpvcHRpb25zKHNjaXBlbj0gOTk5KSAjIHByZXZlbnQgc2NpZW50aWZpYyBub3RhdGlvbg0Kc291cmNlKCJwYWNrYWdlcy5SIikgIyBsb2FkIHBhY2thZ2VzDQpzb3VyY2UoImV2YWxfZnVuY3Rpb25zLlIiKSAjIGltcG9ydCBldmFsdWF0aW9uIGZ1bmN0aW9ucw0KDQpuX3JlcCA8LSAxMDAgIyBkZWZpbmUgbnVtYmVyIG9mIGl0ZXJhdGlvbnMNCmBgYA0KDQojIExpbmRlbiAyMDE2DQoNCiMjIyBJbXBvcnQgc2ltdWxhdGlvbiByZXN1bHRzDQoNCmBgYHtyfQ0KZGF0YW5hbWUgPC0gIkxpbmRlbl8yMDE1Ig0KDQojIGltcG9ydCBzaW11bGF0aW9uIHJlc3VsdHMNCnNpbV9yZXN1bHRzIDwtIHJlYWRSRFMocGFzdGUwKCJzaW1fcmVzdWx0cy9zaW11bGF0aW9uX3Jlc3VsdHNfZmluYWxfIiwgZGF0YW5hbWUsIl8iLCBuX3JlcCwgIi5yZHMiKSkNCg0KIyBkZWZpbmUgY2xhc3NpZmllcnMNCmNsYXNzaWZpZXJzIDwtIG5hbWVzKHNpbV9yZXN1bHRzKQ0KY2xhc3NpZmllcnMgPC0gY2xhc3NpZmllcnNbIShjbGFzc2lmaWVycyAlaW4lIGMoIm91dGNvbWUiLCAidHJ1ZV9lIikpXQ0KY2xhc3NpZmllcnNfY2xlYW4gPC0gYygiYWRhQm9vc3QiLCAiYmFnZ2luZyIgLCJrTk4iLCAiTERBIiwgIk11bHRpbmltaWFsIGxvZ2l0IChnbG1uZXQpIiwgIk11bHRpbmltaWFsIGxvZ2l0IChubmV0KSIsICJNTFBDIiwgIk5haXZlIEJheWVzIChCZXJudWxsaSkiLCAiTmFpdmUgQmF5ZXMgKEdhdXNzaWFuKSIsICJQcm9iYWJpbGl0eSBmb3Jlc3QgKGdyZikiLCAiUHJvYmFiaWxpdHkgZm9yZXN0IChyYW5nZXIpIiwgIlFEQSIsICJTVk0iLCAiWEdCb29zdCIpDQpgYGANCg0KIyMjIENvbXB1dGUgYXZlcmFnZSBldmFsdWF0aW9uIG1ldHJpY3MNCg0KVGhlIHByZWRpY3RlZCBwcm9wZW5zaXR5IHNjb3JlIGFyZSBldmFsdWF0ZWQgdXNpbmcgdGhlIEJyaWVyIHNjb3JlLCB0aGUgUk1TRSBhbmQgdGhlIGxvZ2lzdGljIGxvc3MuDQoNCmBgYHtyfQ0KYnNfbWF0IDwtIG1hdHJpeChOQSwgbnJvdyA9IG5fcmVwLCBuY29sID0gbGVuZ3RoKGNsYXNzaWZpZXJzKSwgZGltbmFtZXMgPSBsaXN0KE5VTEwsIGNsYXNzaWZpZXJzKSkNCmxsX21hdCA8LSBtYXRyaXgoTkEsIG5yb3cgPSBuX3JlcCwgbmNvbCA9IGxlbmd0aChjbGFzc2lmaWVycyksIGRpbW5hbWVzID0gbGlzdChOVUxMLCBjbGFzc2lmaWVycykpDQpybXNlX21hdCA8LSBtYXRyaXgoTkEsIG5yb3cgPSBuX3JlcCwgbmNvbCA9IGxlbmd0aChjbGFzc2lmaWVycyksIGRpbW5hbWVzID0gbGlzdChOVUxMLCBjbGFzc2lmaWVycykpDQoNCmJzX21hdF9vcmFjbGUgPC0gbWF0cml4KE5BLCBucm93PW5fcmVwLCBuY29sID0gMSkNCg0Kbj1sZW5ndGgoc2ltX3Jlc3VsdHMkb3V0Y29tZVtbMV1dJFcpDQpmb3IgKGkgaW4gMTpuX3JlcCl7DQogIGJzX21hdF9vcmFjbGVbaV0gPC0gYnJpZXJfc2NvcmUocHJvYmFiaWxpdGllcz1zaW1fcmVzdWx0c1tbInRydWVfZSJdXVtbaV1dLCANCiAgICAgICAgICAgICAgICAgICAgICBvdXRjb21lPXNpbV9yZXN1bHRzW1sib3V0Y29tZSJdXVtbaV1dJFcsIA0KICAgICAgICAgICAgICAgICAgICAgIGJpbmFyeSA9IEZBTFNFKQ0KDQogIGZvciAoYyBpbiBzZXFfYWxvbmcoY2xhc3NpZmllcnMpKSB7DQogICAgY2xhc3NpZmllciA8LSBjbGFzc2lmaWVyc1tjXQ0KDQogICAgYnNfbWF0W2ksIGNdIDwtIGJyaWVyX3Njb3JlKHByb2JhYmlsaXRpZXM9c2ltX3Jlc3VsdHNbW2NsYXNzaWZpZXJdXVtbaV1dLCANCiAgICAgICAgICAgICAgICAgICAgICBvdXRjb21lPXNpbV9yZXN1bHRzW1sib3V0Y29tZSJdXVtbaV1dJFcsIA0KICAgICAgICAgICAgICAgICAgICAgIGJpbmFyeSA9IEZBTFNFKQ0KICAgIA0KICAgIHJtc2VfbWF0W2ksIGNdIDwtIHNxcnQobWVhbihyb3dNZWFucygoc2ltX3Jlc3VsdHNbW2NsYXNzaWZpZXJdXVtbaV1dIC0gc2ltX3Jlc3VsdHNbWyJ0cnVlX2UiXV1bW2ldXSleMiwgbmEucm0gPSBUUlVFKSwgbmEucm0gPSBUUlVFKSkNCg0KICAgIGxsX21hdFtpLCBjXSA8LSBjcm9zc19lbnRyb3B5KHNpbV9yZXN1bHRzW1tjbGFzc2lmaWVyXV1bW2ldXSwgc2ltX3Jlc3VsdHNbWyJvdXRjb21lIl1dW1tpXV0kVykNCiAgfQ0KfQ0KYGBgDQoNCiMjIyBSZXNoYXBlIGFuZCBhZ2dyZWdhdGUgcmVzdWx0cw0KDQoxLiAgQnJpZXIgc2NvcmUNCg0KYGBge3J9DQpic19sb25nIDwtIGJzX21hdCAlPiUNCiAgYXMuZGF0YS5mcmFtZSgpICU+JQ0KICBwaXZvdF9sb25nZXIoY29scyA9IGV2ZXJ5dGhpbmcoKSkgICU+JQ0KICAjIFNlcGFyYXRlIHRoZSAnbmFtZScgaW50byAnY2xhc3NpZmllcicgYW5kICd2ZXJzaW9uJyBjb2x1bW5zDQogIG11dGF0ZSh2ZXJzaW9uID0gY2FzZV93aGVuKA0KICAgICAgICAgICBzdHJfZGV0ZWN0KG5hbWUsICJeb3ZyXyIpIH4gIm9uZS12cy1yZXN0IiwNCiAgICAgICAgICAgc3RyX2RldGVjdChuYW1lLCAiXm92b18iKSB+ICJvbmUtdnMtb25lIiwNCiAgICAgICAgICAgVFJVRSB+ICJtdWx0aS1jbGFzcyINCiAgICAgICAgICksDQogICAgICAgICBuYW1lID0gZ3N1Yigib3ZyX3xvdm9fIiwgIiIsIG5hbWUpDQogICAgICAgICApDQpic193aWRlIDwtIGJzX2xvbmcgJT4lDQogIGdyb3VwX2J5KHZlcnNpb24sIG5hbWUpICU+JSANCiAgc3VtbWFyaXNlKHZhbHVlID0gcm91bmQobWVhbih2YWx1ZSksIDYzNSkpICU+JSANCiAgc3ByZWFkKGtleSA9IHZlcnNpb24sIHZhbHVlID0gdmFsdWUpIA0KDQprYWJsZShic193aWRlLCBjYXB0aW9uID0gIkJyaWVyIFNjb3JlIikNCmBgYA0KDQoyLiAgUk1TRQ0KDQpgYGB7cn0NCnJtc2VfbG9uZyA8LSBybXNlX21hdCAlPiUNCiAgYXMuZGF0YS5mcmFtZSgpICU+JQ0KICBwaXZvdF9sb25nZXIoY29scyA9IGV2ZXJ5dGhpbmcoKSkgICU+JQ0KICAjIFNlcGFyYXRlIHRoZSAnbmFtZScgaW50byAnY2xhc3NpZmllcicgYW5kICd2ZXJzaW9uJyBjb2x1bW5zDQogIG11dGF0ZSh2ZXJzaW9uID0gY2FzZV93aGVuKA0KICAgICAgICAgICBzdHJfZGV0ZWN0KG5hbWUsICJeb3ZyXyIpIH4gIm9uZS12cy1yZXN0IiwNCiAgICAgICAgICAgc3RyX2RldGVjdChuYW1lLCAiXm92b18iKSB+ICJvbmUtdnMtb25lIiwNCiAgICAgICAgICAgVFJVRSB+ICJtdWx0aS1jbGFzcyINCiAgICAgICAgICksDQogICAgICAgICBuYW1lID0gZ3N1Yigib3ZyX3xvdm9fIiwgIiIsIG5hbWUpDQogICAgICAgICApIA0Kcm1zZV93aWRlIDwtIHJtc2VfbG9uZyAlPiUNCiAgZ3JvdXBfYnkodmVyc2lvbiwgbmFtZSkgJT4lIA0KICBzdW1tYXJpc2UodmFsdWUgPSByb3VuZChtZWFuKHZhbHVlKSwgNCkpICU+JSANCiAgc3ByZWFkKGtleSA9IHZlcnNpb24sIHZhbHVlID0gdmFsdWUpIA0KDQprYWJsZShybXNlX3dpZGUsIGNhcHRpb24gPSAiUk1TRSIpDQpgYGANCg0KMy4gIExvZ2lzdGljIGxvc3MNCg0KYGBge3J9DQpsbF9sb25nIDwtIGxsX21hdCAlPiUNCiAgYXMuZGF0YS5mcmFtZSgpICU+JQ0KICBwaXZvdF9sb25nZXIoY29scyA9IGV2ZXJ5dGhpbmcoKSkgICU+JQ0KICAjIFNlcGFyYXRlIHRoZSAnbmFtZScgaW50byAnY2xhc3NpZmllcicgYW5kICd2ZXJzaW9uJyBjb2x1bW5zDQogIG11dGF0ZSh2ZXJzaW9uID0gY2FzZV93aGVuKA0KICAgICAgICAgICBzdHJfZGV0ZWN0KG5hbWUsICJeb3ZyXyIpIH4gIm9uZS12cy1yZXN0IiwNCiAgICAgICAgICAgc3RyX2RldGVjdChuYW1lLCAiXm92b18iKSB+ICJvbmUtdnMtb25lIiwNCiAgICAgICAgICAgVFJVRSB+ICJtdWx0aS1jbGFzcyINCiAgICAgICAgICksDQogICAgICAgICBuYW1lID0gZ3N1Yigib3ZyX3xvdm9fIiwgIiIsIG5hbWUpDQogICAgICAgICApIA0KbGxfd2lkZSA8LSBsbF9sb25nICU+JQ0KICBncm91cF9ieSh2ZXJzaW9uLCBuYW1lKSAlPiUgDQogIHN1bW1hcmlzZSh2YWx1ZSA9IHJvdW5kKG1lYW4odmFsdWUpLCA0KSkgJT4lIA0KICBzcHJlYWQoa2V5ID0gdmVyc2lvbiwgdmFsdWUgPSB2YWx1ZSkgDQoNCmthYmxlKGxsX3dpZGUsIGNhcHRpb24gPSAiTG9nIExvc3MiKQ0KYGBgDQoNCkV4cG9ydCB0aGUgZmluYWwgcmVzdWx0cy4NCg0KYGBge3J9DQojIFdyaXRlIGJzX3dpZGUgZGF0YSBmcmFtZSBhcyBhbiBFeGNlbCBmaWxlDQp3cml0ZS54bHN4KGJzX3dpZGUsIGZpbGUgPSBwYXN0ZTAoInNpbV9yZXN1bHRzL3Jlc3VsdHNfYnNfIiwgZGF0YW5hbWUsICIueGxzeCIpKQ0KDQojIFdyaXRlIG1zZV93aWRlIGRhdGEgZnJhbWUgYXMgYW4gRXhjZWwgZmlsZQ0Kd3JpdGUueGxzeChybXNlX3dpZGUsIGZpbGUgPSBwYXN0ZTAoInNpbV9yZXN1bHRzL3Jlc3VsdHNfbXNlXyIsIGRhdGFuYW1lLCAiLnhsc3giKSkNCg0KYGBgDQoNCiMjIyBWaXN1YWxpemF0aW9uDQoNCkZvciB0aGUgc3ludGhldGljIGRhdGFzZXRzLCB0aGUgdHJ1ZSBwcm9wZW5zaXR5IHNjb3JlcyBlIGFyZSBrbm93biBhbmQgY2FuIGJlIHV0aWxpemVkIHRvIGVzdGFibGlzaCBhIGJlbmNobWFyayBmb3IgZXZhbHVhdGluZyB0aGUgQnJpZXIgc2NvcmUuIFRoaXMgcmVmZXJlbmNlIEJyaWVyIHNjb3JlIGNhbGN1bGF0ZWQgdXNpbmcgdGhlIGdyb3VuZCB0cnV0aCBlIGlzIGRlbm90ZWQgYXMgQlNfb3JhY2xlLiBCU19vcmFjbGUgaGFzIHBlcmZlY3QgaW5mb3JtYXRpb24gb2YgdGhlIGRhdGEgYW5kIGNhbiBiZSBzZWVuIGFzIGEgbG93ZXIgYm91bmQgZm9yIHRoZSBCcmllciBzY29yZS4gQSBzZWNvbmQgYmVuY2htYXJrIGZvciB0aGUgQnJpZXIgc2NvcmUgaXMgYSBuYcOvdmUgZ3Vlc3MsIHVzaW5nIGVxdWFsIHRyZWF0bWVudCBwcm9wZW5zaXR5IHNjb3JlcyBvZiBlID0gMS9UIGZvciBldmVyeSB0cmVhdG1lbnQgbGV2ZWwgZGVub3RlZCBhcyBCU19uYWl2ZS4NCg0KYGBge3J9DQpLIDwtIGxlbmd0aCh1bmlxdWUoc2ltX3Jlc3VsdHMkb3V0Y29tZVtbMV1dJFcpKSAjIG51bWJlciBvZiB0cmVhdG1lbnRzDQpCU19uYWl2ZSA8LSAoKCgxL0spLTEpXjIgKyAoSy0xKSooKDEvSyleMikpL0sgIyBCU19uYWl2ZQ0KQlNfb3JhY2xlIDwtIG1lYW4oYnNfbWF0X29yYWNsZSkgIyBCU19vcmFjbGUNCiAgDQpwbG90MSA8LSBnZ3Bsb3QoYnNfbG9uZywgYWVzKHggPSB2YWx1ZSwgeSA9IGZjdF9yZXYobmFtZSksIGZpbGwgPSB2ZXJzaW9uKSkgKw0KICBnZW9tX2RlbnNpdHlfcmlkZ2VzKGFscGhhID0gMC45LCBzaG93LmxlZ2VuZCA9IEZBTFNFKSArDQogIGdlb21fdmxpbmUoeGludGVyY2VwdCA9IEJTX25haXZlLCBjb2xvcj0iYmxhY2siLCBsaW5ldHlwZSA9ICJkYXNoZWQiLCBzaXplID0gMC40KSArDQogIGdlb21fdmxpbmUoeGludGVyY2VwdCA9IEJTX29yYWNsZSwgY29sb3I9ImJsYWNrIiwgbGluZXR5cGUgPSAiZGFzaGVkIiwgc2l6ZSA9IDAuNCkgKw0KICBzY2FsZV9maWxsX3BhbGV0dGVlcl9kKCJ0YXlsb1Jzd2lmdDo6bWlkbmlnaHRzQmxvb2RNb29uIiwgZGlyZWN0aW9uID0gMSwgZHluYW1pYyA9IEZBTFNFKSArDQogIGxhYnMoeT0iIiwgeD0iQnJpZXIgU2NvcmUiLCBmaWxsPSIiKSArDQogIHRoZW1lX2J3KCkgKw0KICB4bGltKDAuMjA1LCAwLjI1KSArDQogIHNjYWxlX3lfZGlzY3JldGUobGFiZWxzID0gcmV2KGNsYXNzaWZpZXJzX2NsZWFuKSkgKw0KICBmYWNldF93cmFwKHZhcnModmVyc2lvbikpDQpwbG90MQ0KZ2dzYXZlKHBsb3QxLCB3aWR0aCA9IDI1MCwgaGVpZ2h0ID0gMTI1LCB1bml0cyA9ICJtbSIsDQogICAgICAgZmlsZW5hbWUgPSAicGxvdHMvZGVuczEucG5nIikNCg0KYGBgDQoNCkFub3RoZXIgd2F5IHRvIGFzc2VzcyB0aGUgcXVhbGl0eSBvZiB0aGUgcHJlZGljdGVkIHByb3BlbnNpdHkgc2NvcmVzIGlzIHRvIHBsb3QgdGhlbSBhZ2FpbnN0IHRoZSBncm91bmQgdHJ1dGggaW4gYSBzY2F0dGVyIHBsb3QuDQoNCmBgYHtyfQ0Kb3JhY2xlX2UgPC0gcmVzaGFwZTI6Om1lbHQoc2ltX3Jlc3VsdHNbWyJ0cnVlX2UiXV1bWzFdXSwgaWQudmFycyA9IE5VTEwsIHZhcmlhYmxlLm5hbWUgPSAiY2xhc3MiLCB2YWx1ZS5uYW1lID0gInRydWUiKQ0KZm9yIChjIGluIGNsYXNzaWZpZXJzKXsNCiAgcHJlZGljdGVkX2UgPC0gcmVzaGFwZTI6Om1lbHQoc2ltX3Jlc3VsdHNbW2NdXVtbMV1dLCBpZC52YXJzID0gTlVMTCwgdmFyaWFibGUubmFtZSA9ICJjbGFzcyIsIHZhbHVlLm5hbWUgPSAicHJlZGljdGlvbiIpICU+JSBzZWxlY3QoLWNsYXNzKQ0KDQogIHBsb3RfZGF0YSA8LSBjYmluZChwcmVkaWN0ZWRfZSwgb3JhY2xlX2UpDQogIA0KICBzY2F0dGVyX3Bsb3QgPC0gZ2dwbG90KGRhdGEgPSBwbG90X2RhdGEsYWVzKHkgPSBwcmVkaWN0aW9uLCB4ID0gdHJ1ZSwgY29sb3I9Y2xhc3MpKSArDQogICAgZ2VvbV9wb2ludChhbHBoYSA9IDAuMywgc2l6ZT0xKSArDQogICAgZ2VvbV9hYmxpbmUoaW50ZXJjZXB0ID0gMCwgc2xvcGUgPSAxLCBsaW5ldHlwZSA9ICJkYXNoZWQiLCBjb2xvciA9ICJyZWQiKSArIA0KICAgIHRoZW1lX2J3KCkgKw0KICAgIHNjYWxlX2NvbG9yX21hbnVhbCh2YWx1ZXM9YygiI0Q4OTBBMEZGIiwgIiMyODIwMjBGRiIsICIjRjhCODQwRkYiKSkgKw0KICAgIGxhYnMoDQogICAgICAgICAgeCA9ICJQcmVkaWN0ZWQgUHJvYmFiaWxpdGllcyDDqih3KSIsDQogICAgICAgICAgeSA9ICJUcnVlIFByb2JhaWxpdGllcyBlKHcpIiwNCiAgICAgICAgICBjb2xvciA9ICJXIg0KICAgICAgICAgICN0aXRsZSA9IHBhc3RlMCgiU2ltdWxhdGlvbiByZXN1bHRzIG9mICIsIGMpDQogICAgICAgICkgKw0KICAgIHhsaW0oMCwxKSArIHlsaW0oMCwxKQ0KICANCiAgcGxvdChzY2F0dGVyX3Bsb3QpDQogIGdnc2F2ZShzY2F0dGVyX3Bsb3QsIHdpZHRoID0gMTI1LCBoZWlnaHQgPSA3NSwgdW5pdHMgPSAibW0iLA0KICAgICAgICAgZmlsZW5hbWUgPSBwYXN0ZTAoInBsb3RzL3RydWV2c3ByZWRfIiwgYywgIi5wbmciKSkNCn0NCmBgYA0KDQojIEltYmVucyAyMDE2DQoNCiMjIyBJbXBvcnQgc2ltdWxhdGlvbiByZXN1bHRzDQoNCmBgYHtyfQ0KZGF0YW5hbWUgPC0gIkltYmVuc18yMDE2Ig0KDQpzaW1fcmVzdWx0cyA8LSByZWFkUkRTKHBhc3RlMCgic2ltX3Jlc3VsdHMvc2ltdWxhdGlvbl9yZXN1bHRzX2ZpbmFsXyIsIGRhdGFuYW1lLCJfIiwgbl9yZXAsICIucmRzIikpDQpgYGANCg0KIyMjIENvbXB1dGUgYXZlcmFnZSBldmFsdWF0aW9uIG1ldHJpY3MNCg0KVGhlIHByZWRpY3RlZCBwcm9wZW5zaXR5IHNjb3JlIGFyZSBldmFsdWF0ZWQgdXNpbmcgdGhlIEJyaWVyIHNjb3JlLCB0aGUgUk1TRSBhbmQgdGhlIGxvZ2lzdGljIGxvc3MuDQoNCmBgYHtyfQ0KYnNfbWF0IDwtIG1hdHJpeChOQSwgbnJvdyA9IG5fcmVwLCBuY29sID0gbGVuZ3RoKGNsYXNzaWZpZXJzKSwgZGltbmFtZXMgPSBsaXN0KE5VTEwsIGNsYXNzaWZpZXJzKSkNCmxsX21hdCA8LSBtYXRyaXgoTkEsIG5yb3cgPSBuX3JlcCwgbmNvbCA9IGxlbmd0aChjbGFzc2lmaWVycyksIGRpbW5hbWVzID0gbGlzdChOVUxMLCBjbGFzc2lmaWVycykpDQpybXNlX21hdCA8LSBtYXRyaXgoTkEsIG5yb3cgPSBuX3JlcCwgbmNvbCA9IGxlbmd0aChjbGFzc2lmaWVycyksIGRpbW5hbWVzID0gbGlzdChOVUxMLCBjbGFzc2lmaWVycykpDQoNCmJzX21hdF9vcmFjbGUgPC0gbWF0cml4KE5BLCBucm93PW5fcmVwLCBuY29sID0gMSkNCg0Kbj1sZW5ndGgoc2ltX3Jlc3VsdHMkb3V0Y29tZVtbMV1dJFcpDQpmb3IgKGkgaW4gMTpuX3JlcCl7DQogIGJzX21hdF9vcmFjbGVbaV0gPC0gYnJpZXJfc2NvcmUocHJvYmFiaWxpdGllcz1zaW1fcmVzdWx0c1tbInRydWVfZSJdXVtbaV1dLCANCiAgICAgICAgICAgICAgICAgICAgICBvdXRjb21lPXNpbV9yZXN1bHRzW1sib3V0Y29tZSJdXVtbaV1dJFcsIA0KICAgICAgICAgICAgICAgICAgICAgIGJpbmFyeSA9IEZBTFNFKQ0KDQogIGZvciAoYyBpbiBzZXFfYWxvbmcoY2xhc3NpZmllcnMpKSB7DQogICAgY2xhc3NpZmllciA8LSBjbGFzc2lmaWVyc1tjXQ0KDQogICAgYnNfbWF0W2ksIGNdIDwtIGJyaWVyX3Njb3JlKHByb2JhYmlsaXRpZXM9c2ltX3Jlc3VsdHNbW2NsYXNzaWZpZXJdXVtbaV1dLCANCiAgICAgICAgICAgICAgICAgICAgICBvdXRjb21lPXNpbV9yZXN1bHRzW1sib3V0Y29tZSJdXVtbaV1dJFcsIA0KICAgICAgICAgICAgICAgICAgICAgIGJpbmFyeSA9IEZBTFNFKQ0KICAgIA0KICAgIHJtc2VfbWF0W2ksIGNdIDwtIHNxcnQobWVhbihyb3dNZWFucygoc2ltX3Jlc3VsdHNbW2NsYXNzaWZpZXJdXVtbaV1dIC0gc2ltX3Jlc3VsdHNbWyJ0cnVlX2UiXV1bW2ldXSleMiwgbmEucm0gPSBUUlVFKSwgbmEucm0gPSBUUlVFKSkNCg0KICAgIGxsX21hdFtpLCBjXSA8LSBjcm9zc19lbnRyb3B5KHNpbV9yZXN1bHRzW1tjbGFzc2lmaWVyXV1bW2ldXSwgc2ltX3Jlc3VsdHNbWyJvdXRjb21lIl1dW1tpXV0kVykNCiAgfQ0KfQ0KYGBgDQoNCiMjIyBSZXNoYXBlIGFuZCBhZ2dyZWdhdGUgcmVzdWx0cw0KDQoxLiAgQnJpZXIgc2NvcmUNCg0KYGBge3J9DQpic19sb25nIDwtIGJzX21hdCAlPiUNCiAgYXMuZGF0YS5mcmFtZSgpICU+JQ0KICBwaXZvdF9sb25nZXIoY29scyA9IGV2ZXJ5dGhpbmcoKSkgICU+JQ0KICAjIFNlcGFyYXRlIHRoZSAnbmFtZScgaW50byAnY2xhc3NpZmllcicgYW5kICd2ZXJzaW9uJyBjb2x1bW5zDQogIG11dGF0ZSh2ZXJzaW9uID0gY2FzZV93aGVuKA0KICAgICAgICAgICBzdHJfZGV0ZWN0KG5hbWUsICJeb3ZyXyIpIH4gIm9uZS12cy1yZXN0IiwNCiAgICAgICAgICAgc3RyX2RldGVjdChuYW1lLCAiXm92b18iKSB+ICJvbmUtdnMtb25lIiwNCiAgICAgICAgICAgVFJVRSB+ICJtdWx0aS1jbGFzcyINCiAgICAgICAgICksDQogICAgICAgICBuYW1lID0gZ3N1Yigib3ZyX3xvdm9fIiwgIiIsIG5hbWUpDQogICAgICAgICApDQpic193aWRlIDwtIGJzX2xvbmcgJT4lDQogIGdyb3VwX2J5KHZlcnNpb24sIG5hbWUpICU+JSANCiAgc3VtbWFyaXNlKHZhbHVlID0gcm91bmQobWVhbih2YWx1ZSksIDQpKSAlPiUgDQogIHNwcmVhZChrZXkgPSB2ZXJzaW9uLCB2YWx1ZSA9IHZhbHVlKSANCg0Ka2FibGUoYnNfd2lkZSwgY2FwdGlvbiA9ICJCcmllciBTY29yZSIpDQpgYGANCg0KMi4gIFJNU0UNCg0KYGBge3J9DQpybXNlX2xvbmcgPC0gcm1zZV9tYXQgJT4lDQogIGFzLmRhdGEuZnJhbWUoKSAlPiUNCiAgcGl2b3RfbG9uZ2VyKGNvbHMgPSBldmVyeXRoaW5nKCkpICAlPiUNCiAgIyBTZXBhcmF0ZSB0aGUgJ25hbWUnIGludG8gJ2NsYXNzaWZpZXInIGFuZCAndmVyc2lvbicgY29sdW1ucw0KICBtdXRhdGUodmVyc2lvbiA9IGNhc2Vfd2hlbigNCiAgICAgICAgICAgc3RyX2RldGVjdChuYW1lLCAiXm92cl8iKSB+ICJvbmUtdnMtcmVzdCIsDQogICAgICAgICAgIHN0cl9kZXRlY3QobmFtZSwgIl5vdm9fIikgfiAib25lLXZzLW9uZSIsDQogICAgICAgICAgIFRSVUUgfiAibXVsdGktY2xhc3MiDQogICAgICAgICApLA0KICAgICAgICAgbmFtZSA9IGdzdWIoIm92cl98b3ZvXyIsICIiLCBuYW1lKQ0KICAgICAgICAgKSANCnJtc2Vfd2lkZSA8LSBybXNlX2xvbmcgJT4lDQogIGdyb3VwX2J5KHZlcnNpb24sIG5hbWUpICU+JSANCiAgc3VtbWFyaXNlKHZhbHVlID0gcm91bmQobWVhbih2YWx1ZSksIDQpKSAlPiUgDQogIHNwcmVhZChrZXkgPSB2ZXJzaW9uLCB2YWx1ZSA9IHZhbHVlKSANCg0Ka2FibGUocm1zZV93aWRlLCBjYXB0aW9uID0gIlJNU0UiKQ0KYGBgDQoNCjMuICBMb2dpc3RpYyBsb3NzDQoNCmBgYHtyfQ0KbGxfbG9uZyA8LSBsbF9tYXQgJT4lDQogIGFzLmRhdGEuZnJhbWUoKSAlPiUNCiAgcGl2b3RfbG9uZ2VyKGNvbHMgPSBldmVyeXRoaW5nKCkpICAlPiUNCiAgIyBTZXBhcmF0ZSB0aGUgJ25hbWUnIGludG8gJ2NsYXNzaWZpZXInIGFuZCAndmVyc2lvbicgY29sdW1ucw0KICBtdXRhdGUodmVyc2lvbiA9IGNhc2Vfd2hlbigNCiAgICAgICAgICAgc3RyX2RldGVjdChuYW1lLCAiXm92cl8iKSB+ICJvbmUtdnMtcmVzdCIsDQogICAgICAgICAgIHN0cl9kZXRlY3QobmFtZSwgIl5vdm9fIikgfiAib25lLXZzLW9uZSIsDQogICAgICAgICAgIFRSVUUgfiAibXVsdGktY2xhc3MiDQogICAgICAgICApLA0KICAgICAgICAgbmFtZSA9IGdzdWIoIm92cl98b3ZvXyIsICIiLCBuYW1lKQ0KICAgICAgICAgKSANCmxsX3dpZGUgPC0gbGxfbG9uZyAlPiUNCiAgZ3JvdXBfYnkodmVyc2lvbiwgbmFtZSkgJT4lIA0KICBzdW1tYXJpc2UodmFsdWUgPSByb3VuZChtZWFuKHZhbHVlKSwgNCkpICU+JSANCiAgc3ByZWFkKGtleSA9IHZlcnNpb24sIHZhbHVlID0gdmFsdWUpIA0KDQprYWJsZShsbF93aWRlLCBjYXB0aW9uID0gIkxvZyBMb3NzIikNCmBgYA0KDQpFeHBvcnQgdGhlIGZpbmFsIHJlc3VsdHMuDQoNCmBgYHtyfQ0KIyBXcml0ZSBic193aWRlIGRhdGEgZnJhbWUgYXMgYW4gRXhjZWwgZmlsZQ0Kd3JpdGUueGxzeChic193aWRlLCBmaWxlID0gcGFzdGUwKCJzaW1fcmVzdWx0cy9yZXN1bHRzX2JzXyIsIGRhdGFuYW1lLCAiLnhsc3giKSkNCg0KIyBXcml0ZSBtc2Vfd2lkZSBkYXRhIGZyYW1lIGFzIGFuIEV4Y2VsIGZpbGUNCndyaXRlLnhsc3gocm1zZV93aWRlLCBmaWxlID0gcGFzdGUwKCJzaW1fcmVzdWx0cy9yZXN1bHRzX21zZV8iLCBkYXRhbmFtZSwgIi54bHN4IikpDQoNCmBgYA0KDQojIyMgVmlzdWFsaXphdGlvbg0KDQpgYGB7cn0NCksgPC0gbGVuZ3RoKHVuaXF1ZShzaW1fcmVzdWx0cyRvdXRjb21lW1sxXV0kVykpDQpCU19uYWl2ZSA8LSAoKCgxL0spLTEpXjIgKyAoSy0xKSooKDEvSyleMikpL0sNCkxMX3VuaWYgPC0gLShsb2coMS9LKSArIGxvZygxLSgxL0spKSkNCkJTX29yYWNsZSA8LSBtZWFuKGJzX21hdF9vcmFjbGUpDQogIA0KcGxvdDIgPC0gZ2dwbG90KGJzX2xvbmcsIGFlcyh4ID0gdmFsdWUsIHkgPSBmY3RfcmV2KG5hbWUpLCBmaWxsID0gdmVyc2lvbikpICsNCiAgZ2VvbV9kZW5zaXR5X3JpZGdlcyhhbHBoYSA9IDAuOSwgc2hvdy5sZWdlbmQgPSBGQUxTRSkgKw0KICBnZW9tX3ZsaW5lKHhpbnRlcmNlcHQgPSBCU19uYWl2ZSwgY29sb3I9ImJsYWNrIiwgbGluZXR5cGUgPSAiZGFzaGVkIikgKw0KICBnZW9tX3ZsaW5lKHhpbnRlcmNlcHQgPSBCU19vcmFjbGUsIGNvbG9yPSJibGFjayIsIGxpbmV0eXBlID0gImRhc2hlZCIpICsNCiAgc2NhbGVfZmlsbF9wYWxldHRlZXJfZCgidGF5bG9Sc3dpZnQ6Om1pZG5pZ2h0c0Jsb29kTW9vbiIsIGRpcmVjdGlvbiA9IDEsIGR5bmFtaWMgPSBGQUxTRSkgKw0KICBsYWJzKHk9IiIsIHg9IkJyaWVyIFNjb3JlIiwgZmlsbD0iIikgKw0KICB0aGVtZV9idygpICsNCiAgeGxpbSgwLjIxNSwgMC4yNSkgKw0KICBzY2FsZV95X2Rpc2NyZXRlKGxhYmVscyA9IHJldihjbGFzc2lmaWVyc19jbGVhbikpICsNCiAgZmFjZXRfd3JhcCh2YXJzKHZlcnNpb24pKQ0KcGxvdDINCmdnc2F2ZShwbG90Miwgd2lkdGggPSAyNTAsIGhlaWdodCA9IDEyNSwgdW5pdHMgPSAibW0iLA0KICAgICAgIGZpbGVuYW1lID0gInBsb3RzL2RlbnMyLnBuZyIpDQoNCmBgYA0KDQpgYGB7cn0NCnBpIDwtIHNpbV9yZXN1bHRzJHRydWVfZVtbMV1dDQoNCmJzX29yYWNsZSA8LSBicmllcl9zY29yZShwcm9iYWJpbGl0aWVzID0gcGksIG91dGNvbWUgPSBzaW1fcmVzdWx0c1tbIm91dGNvbWUiXV1bWzFdXSRXKQ0KbGxfb3JhY2xlIDwtIGNyb3NzX2VudHJvcHkocHJvYmFiaWxpdGllcyA9IHBpLCBvdXRjb21lID0gc2ltX3Jlc3VsdHNbWyJvdXRjb21lIl1dW1sxXV0kVykNCg0KcHJlZGljdGVkX2UgPC0gcmVzaGFwZTI6Om1lbHQoc2ltX3Jlc3VsdHNbW2NdXVtbMV1dLCBpZC52YXJzID0gTlVMTCwgdmFyaWFibGUubmFtZSA9ICJjbGFzcyIsIHZhbHVlLm5hbWUgPSAicHJlZGljdGlvbiIpDQoNCmZvciAoYyBpbiBjbGFzc2lmaWVycyl7DQogIG9yYWNsZV9lIDwtIHJlc2hhcGUyOjptZWx0KHBpLCBpZC52YXJzID0gTlVMTCwgdmFyaWFibGUubmFtZSA9ICJjbGFzcyIsIHZhbHVlLm5hbWUgPSAidHJ1ZSIpICU+JSBzZWxlY3QodHJ1ZSkNCiAgcGxvdF9kYXRhIDwtIGNiaW5kKHByZWRpY3RlZF9lLCBvcmFjbGVfZSkNCg0KICBzY2F0dGVyX3Bsb3QgPC0gZ2dwbG90KGRhdGEgPSBwbG90X2RhdGEsYWVzKHkgPSBwcmVkaWN0aW9uLCB4ID0gdHJ1ZSwgY29sb3I9Y2xhc3MpKSArDQogICAgZ2VvbV9wb2ludChhbHBoYSA9IDAuMykgKw0KICAgIGdlb21fYWJsaW5lKGludGVyY2VwdCA9IDAsIHNsb3BlID0gMSwgbGluZXR5cGUgPSAiZGFzaGVkIiwgY29sb3IgPSAicmVkIikgKyANCiAgICBnZW9tX3Ntb290aChjb2xvcj0icmVkIiwgc2UgPSBGQUxTRSwgbGluZXdpZHRoID0gMC43KSsNCiAgICB0aGVtZV9idygpICsNCiAgICBzY2FsZV9jb2xvcl9wYWxldHRlZXJfZCgidGF5bG9Sc3dpZnQ6Om1pZG5pZ2h0c0Jsb29kTW9vbiIsIGRpcmVjdGlvbiA9IDEsIGR5bmFtaWMgPSBGQUxTRSkgKw0KICAgIGxhYnMoDQogICAgICAgICAgeCA9ICJQcmVkaWN0ZWQgUHJvYmFiaWxpdGllcyIsDQogICAgICAgICAgeSA9ICJUcnVlIFByb2JhaWxpdGllcyIsDQogICAgICAgICAgdGl0bGUgPSBwYXN0ZTAoIlNpbXVsYXRpb24gcmVzdWx0cyBvZiAiLCBjKQ0KICAgICAgICApICsNCiAgICB4bGltKDAsMSkgKyB5bGltKDAsMSkNCg0KcGxvdChzY2F0dGVyX3Bsb3QpDQp9DQpgYGANCg0KIyBBY2hhcmt5IDIwMjMNCg0KIyMjIEltcG9ydCBzaW11bGF0aW9uIHJlc3VsdHMNCg0KYGBge3J9DQpkYXRhbmFtZSA8LSAiQWNoYXJreV8yMDIzIg0Kbl9yZXAgPC0gMTAwDQoNCnNpbV9yZXN1bHRzIDwtIHJlYWRSRFMocGFzdGUwKCJzaW1fcmVzdWx0cy9zaW11bGF0aW9uX3Jlc3VsdHNfb3ZvXyIsIGRhdGFuYW1lLCJfIiwgbl9yZXAsICIucmRzIikpDQpgYGANCg0KIyMjIENvbXB1dGUgYXZlcmFnZSBldmFsdWF0aW9uIG1ldHJpY3MNCg0KVGhlIHByZWRpY3RlZCBwcm9wZW5zaXR5IHNjb3JlIGFyZSBldmFsdWF0ZWQgdXNpbmcgdGhlIEJyaWVyIHNjb3JlIGFuZCB0aGUgbG9naXN0aWMgbG9zcy4gQXMgZm9yIHRoZSBzZW1pLXN5bnRoZXRpYyBkYXRhc2V0IHRoZSBncm91bmQgdHJ1dGggb2YgdGhlIHByb3BlbnNpdHkgc2NvcmUgaXMgbm90IGtub3csIHRoZSBSTVNFIGNhbm5vdCBiZSBjYWxjdWxhdGVkLg0KDQpgYGB7cn0NCmJzX21hdCA8LSBtYXRyaXgoTkEsIG5yb3cgPSBuX3JlcCwgbmNvbCA9IGxlbmd0aChjbGFzc2lmaWVycyksIGRpbW5hbWVzID0gbGlzdChOVUxMLCBjbGFzc2lmaWVycykpDQpsbF9tYXQgPC0gbWF0cml4KE5BLCBucm93ID0gbl9yZXAsIG5jb2wgPSBsZW5ndGgoY2xhc3NpZmllcnMpLCBkaW1uYW1lcyA9IGxpc3QoTlVMTCwgY2xhc3NpZmllcnMpKQ0Kbj1sZW5ndGgoc2ltX3Jlc3VsdHMkb3V0Y29tZVtbMV1dJFcpDQpmb3IgKGMgaW4gc2VxX2Fsb25nKGNsYXNzaWZpZXJzKSkgew0KICBjbGFzc2lmaWVyIDwtIGNsYXNzaWZpZXJzW2NdDQogIGZvciAoaSBpbiAxOm5fcmVwKXsNCiAgICBicyA8LSBicmllcl9zY29yZShzaW1fcmVzdWx0c1tbY2xhc3NpZmllcl1dW1tpXV0sIHNpbV9yZXN1bHRzW1sib3V0Y29tZSJdXVtbaV1dJFcsIGJpbmFyeSA9IEZBTFNFKQ0KICAgIGJzX21hdFtpLCBjXSA8LSBicw0KICAgIA0KICAgIGxsIDwtIGNyb3NzX2VudHJvcHkoc2ltX3Jlc3VsdHNbW2NsYXNzaWZpZXJdXVtbaV1dLCBzaW1fcmVzdWx0c1tbIm91dGNvbWUiXV1bW2ldXSRXKQ0KICAgIGxsX21hdFtpLCBjXSA8LSBsbA0KICB9DQp9DQpgYGANCg0KIyMjIFJlc2hhcGUgYW5kIGFnZ3JlZ2F0ZSByZXN1bHRzDQoNCjEuICBCcmllciBzY29yZQ0KDQpgYGB7cn0NCmJzX2xvbmcgPC0gYnNfbWF0ICU+JQ0KICBhcy5kYXRhLmZyYW1lKCkgJT4lDQogIHBpdm90X2xvbmdlcihjb2xzID0gZXZlcnl0aGluZygpKSAgJT4lDQogICMgU2VwYXJhdGUgdGhlICduYW1lJyBpbnRvICdjbGFzc2lmaWVyJyBhbmQgJ3ZlcnNpb24nIGNvbHVtbnMNCiAgbXV0YXRlKHZlcnNpb24gPSBjYXNlX3doZW4oDQogICAgICAgICAgIHN0cl9kZXRlY3QobmFtZSwgIl5vdnJfIikgfiAib25lLXZzLXJlc3QiLA0KICAgICAgICAgICBzdHJfZGV0ZWN0KG5hbWUsICJeb3ZvXyIpIH4gIm9uZS12cy1vbmUiLA0KICAgICAgICAgICBUUlVFIH4gIm11bHRpLWNsYXNzIg0KICAgICAgICAgKSwNCiAgICAgICAgIG5hbWUgPSBnc3ViKCJvdnJffG92b18iLCAiIiwgbmFtZSkNCiAgICAgICAgICkNCmJzX3dpZGUgPC0gYnNfbG9uZyAlPiUNCiAgZ3JvdXBfYnkodmVyc2lvbiwgbmFtZSkgJT4lIA0KICBzdW1tYXJpc2UodmFsdWUgPSByb3VuZChtZWFuKHZhbHVlKSwgNCkpICU+JSANCiAgc3ByZWFkKGtleSA9IHZlcnNpb24sIHZhbHVlID0gdmFsdWUpIA0KDQprYWJsZShic193aWRlLCBjYXB0aW9uID0gIkJyaWVyIFNjb3JlIikNCmBgYA0KDQoyLiAgTG9naXN0aWMgbG9zcw0KDQpgYGB7cn0NCmxsX2xvbmcgPC0gbGxfbWF0ICU+JQ0KICBhcy5kYXRhLmZyYW1lKCkgJT4lDQogIHBpdm90X2xvbmdlcihjb2xzID0gZXZlcnl0aGluZygpKSAgJT4lDQogICMgU2VwYXJhdGUgdGhlICduYW1lJyBpbnRvICdjbGFzc2lmaWVyJyBhbmQgJ3ZlcnNpb24nIGNvbHVtbnMNCiAgbXV0YXRlKHZlcnNpb24gPSBjYXNlX3doZW4oDQogICAgICAgICAgIHN0cl9kZXRlY3QobmFtZSwgIl5vdnJfIikgfiAib25lLXZzLXJlc3QiLA0KICAgICAgICAgICBzdHJfZGV0ZWN0KG5hbWUsICJeb3ZvXyIpIH4gIm9uZS12cy1vbmUiLA0KICAgICAgICAgICBUUlVFIH4gIm11bHRpLWNsYXNzIg0KICAgICAgICAgKSwNCiAgICAgICAgIG5hbWUgPSBnc3ViKCJvdnJffG92b18iLCAiIiwgbmFtZSkNCiAgICAgICAgICkgDQpsbF93aWRlIDwtIGxsX2xvbmcgJT4lDQogIGdyb3VwX2J5KHZlcnNpb24sIG5hbWUpICU+JSANCiAgc3VtbWFyaXNlKHZhbHVlID0gcm91bmQobWVhbih2YWx1ZSksIDQpKSAlPiUgDQogIHNwcmVhZChrZXkgPSB2ZXJzaW9uLCB2YWx1ZSA9IHZhbHVlKSANCg0Ka2FibGUobGxfd2lkZSwgY2FwdGlvbiA9ICJMb2cgTG9zcyIpDQpgYGANCg0KRXhwb3J0IHRoZSBmaW5hbCByZXN1bHRzLg0KDQpgYGB7cn0NCiMgV3JpdGUgYnNfd2lkZSBkYXRhIGZyYW1lIGFzIGFuIEV4Y2VsIGZpbGUNCndyaXRlLnhsc3goYnNfd2lkZSwgZmlsZSA9IHBhc3RlMCgic2ltX3Jlc3VsdHMvcmVzdWx0c19ic18iLCBkYXRhbmFtZSwgIi54bHN4IikpDQpgYGANCg0KIyMjIFZpc3VhbGl6YXRpb24NCg0KYGBge3J9DQpLIDwtIGxlbmd0aCh1bmlxdWUoc2ltX3Jlc3VsdHMkb3V0Y29tZVtbMV1dJFcpKQ0KQlNfb3JhY2xlIDwtICgoKDEvSyktMSleMiArIChLLTEpKigoMS9LKV4yKSkvSw0KTExfdW5pZiA8LSAtKGxvZygxL0spICsgbG9nKDEtKDEvSykpKQ0KDQpwbG90MyA8LSBnZ3Bsb3QoYnNfbG9uZywgYWVzKHggPSB2YWx1ZSwgeSA9IGZjdF9yZXYobmFtZSksIGZpbGwgPSB2ZXJzaW9uKSkgKw0KICBnZW9tX2RlbnNpdHlfcmlkZ2VzKGFscGhhID0gMC45LCBzaG93LmxlZ2VuZCA9IEZBTFNFKSArDQogIGdlb21fdmxpbmUoeGludGVyY2VwdCA9IEJTX29yYWNsZSwgY29sb3I9ImJsYWNrIiwgbGluZXR5cGUgPSAiZGFzaGVkIikgKw0KICBzY2FsZV9maWxsX3BhbGV0dGVlcl9kKCJ0YXlsb1Jzd2lmdDo6bWlkbmlnaHRzQmxvb2RNb29uIiwgZGlyZWN0aW9uID0gMSwgZHluYW1pYyA9IEZBTFNFKSArDQogIGxhYnMoeT0iIiwgeD0iQnJpZXIgU2NvcmUiLCBmaWxsPSIiKSArDQogIHRoZW1lX2J3KCkgKw0KICBzY2FsZV94X2NvbnRpbnVvdXMoYnJlYWtzID0gYygwLjA3MSwgMC4wNzE1LCAwLjA3MiksIGxpbWl0cyA9IGMoMC4wNzA4LCAwLjA3MjEpKSArDQogIHNjYWxlX3lfZGlzY3JldGUobGFiZWxzID0gcmV2KGNsYXNzaWZpZXJzX2NsZWFuKSkgKw0KICBmYWNldF93cmFwKHZhcnModmVyc2lvbikpDQpwbG90Mw0KZ2dzYXZlKHBsb3QzLCB3aWR0aCA9IDI1MCwgaGVpZ2h0ID0gMTI1LCB1bml0cyA9ICJtbSIsDQogICAgICAgZmlsZW5hbWUgPSAicGxvdHMvZGVuczMucG5nIikNCg0KYGBgDQoNCmBgYHtyfQ0KZm9yIChjIGluIGNsYXNzaWZpZXJzKXsNCiAgcGxvdChtYWtlX2NhbGlidGF0aW9uX3Bsb3QocHJvYmFiaWxpdGllcyA9IHNpbV9yZXN1bHRzW1tjXV1bWzFdXSwgb3V0Y29tZSA9IHNpbV9yZXN1bHRzJG91dGNvbWVbWzFdXSRXLCBtZXRob2QgPSBjKSkNCn0NCmBgYA0K